{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "4acb3450",
   "metadata": {},
   "source": [
    "## PPO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "88e76a36",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-11T08:03:32.225514Z",
     "start_time": "2024-05-11T08:03:27.590866Z"
    }
   },
   "outputs": [],
   "source": [
    "import gym\n",
    "import numpy as np\n",
    "from IPython import display\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "389eefe7",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-11T08:03:33.542268Z",
     "start_time": "2024-05-11T08:03:33.537254Z"
    }
   },
   "outputs": [],
   "source": [
    "class GymHelper:\n",
    "    def __init__(self,env,figsize=(3,3)):\n",
    "        self.env=env\n",
    "        self.figsize=figsize\n",
    "        plt.figure(figsize=figsize)\n",
    "        self.img=plt.imshow(env.render())\n",
    "    def render(self,title=None):\n",
    "        img_data=self.env.render()\n",
    "        self.img.set_data(img_data)\n",
    "        display.display(plt.gcf())\n",
    "        display.clear_output(wait=True)\n",
    "        if title:\n",
    "            plt.title(title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ee3a2d54",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-11T08:05:46.109992Z",
     "start_time": "2024-05-11T08:05:42.990131Z"
    }
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "from tqdm import *\n",
    "import collections\n",
    "import time\n",
    "import random\n",
    "import sys\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a8c5b71b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-11T08:11:18.635749Z",
     "start_time": "2024-05-11T08:11:18.628060Z"
    }
   },
   "outputs": [],
   "source": [
    "#策略模型，给定状态生成各个动作的概率\n",
    "class Policymodel(nn.Module):\n",
    "    def __init__(self,input_dim,output_dim):\n",
    "        super(Policymodel,self).__init__()\n",
    "        self.input_dim=input_dim\n",
    "        self.output_dim=output_dim\n",
    "        self.fc=nn.Sequential(\n",
    "            nn.Linear(self.input_dim,128),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(128,128),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(128,self.output_dim),\n",
    "            nn.Softmax(dim=1)\n",
    "        )\n",
    "        #dueling networks\n",
    "    def forward(self,state):\n",
    "        action_prob=self.fc(state)\n",
    "        return action_prob\n",
    "#价值模型，给定状态的估计值\n",
    "class Valuemodel(nn.Module):\n",
    "    def __init__(self,input_dim):\n",
    "        super(Valuemodel,self).__init__()\n",
    "        self.input_dim=input_dim\n",
    "        #self.output_dim=output_dim\n",
    "        self.fc=nn.Sequential(\n",
    "            nn.Linear(self.input_dim,128),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(128,128),\n",
    "            nn.ReLU(),\n",
    "#             nn.Linear(128,self.output_dim),\n",
    "            nn.Linear(128,1)\n",
    "        )\n",
    "        #dueling networks\n",
    "    def forward(self,x):\n",
    "        value=self.fc(x)\n",
    "        return value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "251b52cc",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-11T08:16:39.194305Z",
     "start_time": "2024-05-11T08:16:39.188283Z"
    }
   },
   "outputs": [],
   "source": [
    "class PPO:\n",
    "    def __init__(self,env,lr=0.001,gamma=0.99,lamda=0.95,eps=0.2,epochs=20):\n",
    "        self.env=env\n",
    "        self.lr=lr\n",
    "        self.gamma=gamma\n",
    "        self.lamda=lamda\n",
    "        self.eps=eps\n",
    "        self.epochs=epochs\n",
    "        #判断可用设备是CPU与GPU\n",
    "        self.device=torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "        #定义策略网络与价值网络\n",
    "        self.policy_model=Policymodel(env.observation_space.shape[0],env.action_space.n).to(self.device)\n",
    "        self.value_model=Valuemodel(env.observation_space.shape[0]).to(self.device)\n",
    "        self.policy_optimizer=torch.optim.Adam(self.policy_model.parameters(),lr=lr)\n",
    "        self.value_optimizer=torch.optim.Adam(self.value_model.parameters(),lr=lr)\n",
    "    def choose_action(self,state):\n",
    "        state=torch.FloatTensor(np.array([state])).to(self.device)\n",
    "        with torch.no_grad():\n",
    "            action_prob=self.policy_model(state)\n",
    "        c=torch.distributions.Categorical(action_prob)\n",
    "        action=c.sample()\n",
    "        return action\n",
    "    def calc_advantage(self,td_delta):\n",
    "        td_delta=td_delta.cpu().detach().numpy()\n",
    "        #初始化\n",
    "        advantage=0\n",
    "        advantage_list=[]\n",
    "        "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
